{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using ReinforcementLearningAnIntroduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Agent{QBasedPolicy{DifferentialTDLearner{TabularApproximator{2,Array{Float64,2}}},EpsilonGreedyExplorer{:linear,false,Random.MersenneTwister}},EpisodicCompactSARTSATrajectory{Tuple{Int64,Int64,Float32,Bool,Int64,Int64},Tuple{Array{Float32,1},Array{Bool,1},Array{Int64,1},Array{Int64,1}}},ReinforcementLearningBase.DefaultPlayer}(QBasedPolicy{DifferentialTDLearner{TabularApproximator{2,Array{Float64,2}}},EpsilonGreedyExplorer{:linear,false,Random.MersenneTwister}}(DifferentialTDLearner{TabularApproximator{2,Array{Float64,2}}}(TabularApproximator{2,Array{Float64,2}}([0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]), 0.01, 0.01, 0.0, 0), EpsilonGreedyExplorer{:linear,false,Random.MersenneTwister}(0.1, 1.0, 0, 0, 1, Random.MersenneTwister(UInt32[0x18e6923b, 0x1423e0bb, 0xed5cb620, 0x200471d7], Random.DSFMT.DSFMT_state(Int32[1066631840, 1073270803, 392812027, 1072842938, -265406330, 1073704675, -1651941256, 1073738140, 404002557, 1073266405  …  -497468239, 1073712303, -1742105735, 1073022714, 1249485821, -1288535463, 1938043466, 1733895721, 382, 0]), [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0  …  0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], UInt128[0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000  …  0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000, 0x00000000000000000000000000000000], 1002, 0))), NamedTuple{(:state, :action, :reward, :terminal, :next_state, :next_action),Tuple{Int64,Int64,Float32,Bool,Int64,Int64}}[], ReinforcementLearningBase.DefaultPlayer())"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = AccessControlEnv()\n",
    "nstates = length(get_observation_space(env))\n",
    "nactions = length(get_action_space(env))\n",
    "agent = Agent(\n",
    "    policy=QBasedPolicy(\n",
    "        learner=DifferentialTDLearner(\n",
    "            approximator=TabularApproximator(n_state=nstates, n_action=nactions),\n",
    "            α=0.01,\n",
    "            β=0.01,\n",
    "        ),\n",
    "        explorer=EpsilonGreedyExplorer(0.1)\n",
    "    ),\n",
    "    trajectory=EpisodicCompactSARTSATrajectory()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EmptyHook()"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run(agent, env, StopAfterStep(2*10^6; is_show_progress=false))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip4700\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip4700)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip4701\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip4700)\" d=\"\n",
       "M141.865 1487.47 L2352.76 1487.47 L2352.76 47.2441 L141.865 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip4702\">\n",
       "    <rect x=\"141\" y=\"47\" width=\"2212\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  436.187,1487.47 436.187,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  899.686,1487.47 899.686,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1363.19,1487.47 1363.19,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1826.68,1487.47 1826.68,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2290.18,1487.47 2290.18,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  141.865,1289.15 2352.76,1289.15 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  141.865,1039.54 2352.76,1039.54 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  141.865,789.923 2352.76,789.923 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  141.865,540.309 2352.76,540.309 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  141.865,290.695 2352.76,290.695 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  141.865,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  141.865,1487.47 141.865,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  436.187,1487.47 436.187,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  899.686,1487.47 899.686,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1363.19,1487.47 1363.19,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1826.68,1487.47 1826.68,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2290.18,1487.47 2290.18,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  141.865,1289.15 168.395,1289.15 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  141.865,1039.54 168.395,1039.54 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  141.865,789.923 168.395,789.923 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  141.865,540.309 168.395,540.309 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  141.865,290.695 168.395,290.695 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 436.187, 1541.47)\" x=\"436.187\" y=\"1541.47\">2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 899.686, 1541.47)\" x=\"899.686\" y=\"1541.47\">4</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1363.19, 1541.47)\" x=\"1363.19\" y=\"1541.47\">6</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1826.68, 1541.47)\" x=\"1826.68\" y=\"1541.47\">8</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2290.18, 1541.47)\" x=\"2290.18\" y=\"1541.47\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 117.865, 1306.65)\" x=\"117.865\" y=\"1306.65\">-3</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 117.865, 1057.04)\" x=\"117.865\" y=\"1057.04\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 117.865, 807.423)\" x=\"117.865\" y=\"807.423\">3</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 117.865, 557.809)\" x=\"117.865\" y=\"557.809\">6</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 117.865, 308.195)\" x=\"117.865\" y=\"308.195\">9</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  204.437,1442.7 436.187,1246.57 667.936,1070.77 899.686,939.746 1131.44,820.371 1363.19,734.136 1594.93,676.397 1826.68,962.552 2058.43,1042.31 2290.18,1038.71 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  204.437,1446.71 436.187,1216.46 667.936,1073.22 899.686,919.115 1131.44,771.511 1363.19,657.183 1594.93,579.683 1826.68,901.621 2058.43,1038.35 2290.18,1039.54 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#3da44d; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  204.437,1410.21 436.187,1109.92 667.936,883.587 899.686,758.141 1131.44,591.601 1363.19,471.483 1594.93,418.759 1826.68,839.817 2058.43,1034.34 2290.18,1039.54 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4702)\" style=\"stroke:#c271d2; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  204.437,1095.09 436.187,802.939 667.936,585.189 899.686,407.054 1131.44,283.438 1363.19,157.059 1594.93,88.0053 1826.68,782.601 2058.43,1028.72 2290.18,1039.54 \n",
       "  \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip4700)\" d=\"\n",
       "M1774.58 1343.47 L2280.76 1343.47 L2280.76 1041.07 L1774.58 1041.07  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1774.58,1343.47 2280.76,1343.47 2280.76,1041.07 1774.58,1041.07 1774.58,1343.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1798.58,1101.55 1942.58,1101.55 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1966.58, 1119.05)\" x=\"1966.58\" y=\"1119.05\">priority = 1.0</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1798.58,1162.03 1942.58,1162.03 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1966.58, 1179.53)\" x=\"1966.58\" y=\"1179.53\">priority = 2.0</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#3da44d; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1798.58,1222.51 1942.58,1222.51 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1966.58, 1240.01)\" x=\"1966.58\" y=\"1240.01\">priority = 4.0</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip4700)\" style=\"stroke:#c271d2; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1798.58,1282.99 1942.58,1282.99 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip4700)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1966.58, 1300.49)\" x=\"1966.58\" y=\"1300.49\">priority = 8.0</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = plot(legend=:bottomright)\n",
    "for i in 1:length(AccessControl.PRIORITIES)\n",
    "    plot!(\n",
    "        [agent.policy.learner.approximator(AccessControl.TRANSFORMER[(CartesianIndex(n+1, i))]) |> maximum\n",
    "            for n in 1:AccessControl.N_SERVERS],\n",
    "        label=\"priority = $(AccessControl.PRIORITIES[i])\")\n",
    "end\n",
    "p"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.3.0",
   "language": "julia",
   "name": "julia-1.3"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
